package com.cnap.multiclouldmgmt.service.impl;

import com.alibaba.fastjson.JSON;
import com.cnap.multiclouldmgmt.model.enums.CloudVendorsEnum;
import com.cnap.multiclouldmgmt.model.knowledge.RunInjectRetrunModel;
import com.cnap.multiclouldmgmt.model.ssh.SshConnection;
import com.cnap.multiclouldmgmt.model.ssh.SshResponse;
import com.cnap.multiclouldmgmt.model.vm.VmModel;
import com.cnap.multiclouldmgmt.service.CnapSshService;
import com.cnap.multiclouldmgmt.util.CnapCloudMgmtConstants;
import com.cnap.multiclouldmgmt.util.DateUtils;
import com.cnap.multiclouldmgmt.util.SshUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import java.util.Date;
import java.util.List;

/**
 * 平台安装
 *
 * ## 环境要求
 * 服务器要求：
 * `Ubuntu 18.04 X86_64`
 *
 * 配置建议不低于如下：
 *
 * - 4 CPU
 *
 * - 16G内存
 *
 * - 200G硬盘（根目录下）
 *
 * - 单网卡或者多网卡
 *
 * - 支持systemd/systemctl
 *
 *
 *
 *
 * ## 纳管集群安装
 *
 * ### 1. 下载安装包和镜像包
 *
 * 下载安装包：CNAP-edge-x86.tar.gz
 * 将安装包通过ftp等工具，上传到待部署服务器上，本操作手册以安装在/home目录下为例，下同。
 *
 * ```
 * cd /home
 * tar -xvf CNAP-edge-x86.tar.gz
 * ```
 *
 * ### 2. 安装依赖软件（pip3/ansible/sshpaas）
 *
 *    ```
 * cd /home/CNAP-edge-x86
 * bash tools_install_offline.sh
 *    ```
 *
 * ### 3. 准备 CNAP 安装环境
 *
 * 1. 配置秘钥文件及免登录部署
 *    ```
 *    sshpass -p '<Master-Root-Password>' ssh-copy-id -p <ssh-port> -o StrictHostKeyChecking=no root@<Master-IP>
 *    sshpass -p '<Worker1-Root-Password>' ssh-copy-id -p <ssh-port> -o StrictHostKeyChecking=no root@<Worker1-IP>
 *    sshpass -p '<Worker2-Root-Password>' ssh-copy-id -p <ssh-port> -o StrictHostKeyChecking=no root@<Worker2-IP>
 *    ```
 *
 *    注：Master-IP 为待部署集群的master节点IP，Worker-IP为待部署集群的worker节点IP
 *
 *    ​        Master-Root-Password  为对应的节点的root用户登录密码
 *
 * ​        ssh-port  为ssh登录的端口，默认为22
 *
 *
 *
 * 2. 配置安装参数 hosts-muno
 *
 *    修改/home/CNAP-edge-x86/install 文件夹里的hosts-muno，配置待部署的节点IP
 *    ```
 *    [master]
 *    xxx.xxx.xxx.xxx
 *
 *    [worker]
 *    xxx.xxx.xxx.xxx
 *    xxx.xxx.xxx.xxx
 *    ```
 *
 *    注：1）master表示主master节点；
 *
 *    ​              worker表示node节点；
 *
 *    ​        2）如果没有该节点，如没有masterSlave节点或worker节点，则留空即可
 *
 *
 *
 * 3. 配置安装参数 default-var.yml （若不需要使用公网IP）
 *    修改/home/CNAP-edge-x86/install/default-var.yml
 *
 *    ```
 *    # 修改为映射的公网IP
 *    PORTAL_IP: xxx.xxx.xxx.xxx
 *
 *    # 修改为算网平台的IP
 *    MECM_IP: xxx.xxx.xxx.xxx
 *
 *    # 若系统盘（根目录）下空间不足，则可修改docker镜像存储路径至其他目录下
 *    DOCKER_ROOT: "/var/lib/docker"
 *    ```
 *
 * ### 4. 安装 CNAP
 *    ```
 * cd /home/CNAP-edge-x86/install
 *
 * ansible-playbook -i hosts-muno cnap_edge_muno_install.yml
 *    ```
 *
 * ## 附1  常见问题
 *
 * 1. Harbor的默认虚拟网段（172.17.xx.xx）与PC网段冲突，导致PC无法连接服务器/打开页面
 *
 *    此种情况需要手动重启一次harbor（以下执行通过bmc等方式连接至服务器）
 *
 *    ```
 *    cd /home/kubernetes/harbor/harbor
 *    docker-compose down
 *    docker-compose up -d
 *    ```
 *
 *
 *
 * ## 附2  增加/减少worker节点
 *
 *    平台安装完成后，可以增加或减少K8S集群中的worker节点，以下操作以中心节点为例，边缘节点操作类似。（所有操作均在master节点执行）
 *
 * 1. 增加worker节点
 *
 *    1. 增加单个节点
 *
 *       ```
 *       1) 修改host-muno文件，添加要增加的worker节点，如下172.171.11.37即为要增加的worker节点
 *
 *       [worker]
 *       172.171.11.47
 *       172.171.11.37
 *
 *       2) 执行安装，使用--limit参数来指定要新增的worker节点
 *       ansible-playbook -i hosts-muno cnap_all_muno_install.yml --limit=172.11.11.37
 *       ```
 *
 *    2. 增加多个节点
 *
 *       ```
 *       1) 如果需要一次性增加多个worker节点，则将需要增加的worker节点的IP写入一个文件，如新建workers.txt并写入:
 *       vi  workers.txt
 *
 *       172.171.11.37
 *       172.171.11.38
 *
 *       然后执行：
 *       ansible-playbook -i hosts-muno cnap_all_muno_install.yml --limit=@workers.txt
 *       ```
 *
 * 2. 减少worker节点
 *
 *    ```
 *    1) 查看当前集群node信息，获取要删除的worker节点的名称<WORKER-NODE-NAME>
 *       kubectl get nodes
 *
 * 	  ansible-playbook -i hosts-muno cnap_edge_muno_install.yml
 *
 *
 *    2) 删除节点
 *    kubectl cordon <WORKER-NODE-NAME>
 *    kubectl drain <WORKER-NODE-NAME> --delete-local-data  --ignore-daemonsets --force
 *    kubectl delete nodes <WORKER-NODE-NAME>
 *    ansible-playbook -i hosts-muno cnap_all_muno_install.yml --limit=172.171.11.37
 *
 *    注：若kubectl drains时提示Cannot evict pod as it would violate the pod's disruption budget，则 kubectl delete pod xxx，删除对应的pod后重新执行即可
 *
 *    3) 修改host-muno文件，删除要减少的worker节点的信息
 *    ```
 */
@Service("CnapSshServiceImpl")
public class CnapSshServiceImpl implements CnapSshService {

    private final static Logger LOGGER = LoggerFactory.getLogger(CnapSshServiceImpl.class);

    //中心服务器地址ip
    @Value("${center.ip}")
    private  String centerIp;

    @Value("${center.passwd}")
    private  String centerPasswd;

    @Value("${center.packagePath}")
    private  String centerPackagePath;

    @Value("${center.packageName}")
    private  String installPackageName;

    @Value("${center.packageGpuName}")
    private  String installPackageGpuName;

    @Value("${center.untarPackageName}")
    private  String untarPackageName;

    @Value("${center.untarPackageGpuName}")
    private  String untarPackageGpuName;

    @Value("${center.packageMd5}")
    private  String md5Str;

    @Value("${center.packageGpuMd5}")
    private  String md5StrGpu;

    //中心docker服务器地址ip-高可用01
    @Value("${docker.url}")
    private String dockerMasterUrl;


    //部署混淆工具的地址
    @Value("${docker.errorPath}")
    private String errorInjectPackagePath;

    @Value("${docker.errorPackageName}")
    private  String errorInjectPackageName;

    @Value("${docker.centerErrorPackagePath}")
    private  String centerInjectPackagePath;

    @Value("${docker.errorToolsInstallCmd}")
    private  String errorToolsInstallCmd;


    private static final String ROOT_NAME = "root";


    /**
     *
     * @param masterModel
     * @return
     * @throws Exception
     */
    @Override
    public boolean getPackageNormal(VmModel masterModel) throws Exception {

        SshConnection conn  = new SshConnection(ROOT_NAME,masterModel.getPasswd(),masterModel.getExternalIp());

        String packageName = installPackageName;
        if (StringUtils.equalsIgnoreCase(masterModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)){
            packageName = installPackageGpuName;
        }

        //获取md5
        String cmd = "md5sum /home/"+packageName;
        SshResponse response = SshUtils.runCommand(conn,cmd);
        if (response.getReturnCode() != 0){
            LOGGER.error("getPackageNormal fail targetIp=" + masterModel.getExternalIp() + " errorMsg=" + response.getErrOutput()
                    + " cmd="+ cmd);
        }

        String rightMd5Str = md5Str;
        if (StringUtils.equalsIgnoreCase(masterModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)){
            rightMd5Str = md5StrGpu;
        }

        if (response.getStdOutput().contains(rightMd5Str)){
            LOGGER.error("getPackageNormal md5 success ip={} output={}",masterModel.getExternalIp(),response.getStdOutput());
            return true;
        }else{
            LOGGER.error("getPackageNormal md5 fail ip={} output={}",masterModel.getExternalIp(),response.getStdOutput());
        }
        return false;
    }

    /**
     *
     * @param vmModel
     * @throws Exception
     */
    @Override
    public void copyMasterInstallPackage(VmModel vmModel) throws Exception {
        SshConnection conn  = new SshConnection(ROOT_NAME,centerPasswd,centerIp);
        VmModel masterModel = new VmModel();
        masterModel.setExternalIp(centerIp);
        masterModel.setPasswd(centerPasswd);

        //ut测试绕过
        if (!StringUtils.equals(vmModel.getUserName(), CnapCloudMgmtConstants.TEST_ADMIN)) {
            //删除安装包
            delExternalIpNoPasswdSsh(masterModel,vmModel);
            setNoPassWdCopy(ROOT_NAME, vmModel.getExternalIp(), vmModel.getPasswd());
        }

        String packageName = installPackageName;
        if (StringUtils.equalsIgnoreCase(vmModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)){
            packageName = installPackageGpuName;
        }
        //拷贝安装包
        try {
            String cmd = "sshpass -p '" + vmModel.getPasswd()
                    + "' scp " + centerPackagePath + packageName + " " + ROOT_NAME + "@"
                    + vmModel.getExternalIp() + ":/home";
            SshResponse response = SshUtils.runCommand(conn, cmd);
            if (response.getReturnCode() != 0) {
                LOGGER.error("copyMasterInstallPackage fail targetIp=" + vmModel.getExternalIp() + " errorMsg=" + response.getErrOutput()
                        + " cmd=" + cmd);
            }
        }catch (Throwable e){
            LOGGER.error("copyMasterInstallPackage error targetIp={}",vmModel.getExternalIp(),e );
        }
    }

    /**
     *
     * @param userName
     * @param tagetIp
     * @param targetPassWd
     * @throws Exception
     */
    @Override
    public void setNoPassWdCopy(String userName,String tagetIp, String targetPassWd) throws Exception {

        setNoPassWdCopy(centerIp,centerPasswd,userName,tagetIp,targetPassWd);
    }

    /**
     *
     * @param sourceIp
     * @param sourcePassWd
     * @param userName
     * @param targetIp
     * @param targetPassWd
     * @throws Exception
     */
    @Override
    public void setNoPassWdCopy(String sourceIp, String sourcePassWd,String userName,String targetIp, String targetPassWd) throws Exception {

        SshConnection conn  = new SshConnection(ROOT_NAME,sourcePassWd,sourceIp);
        //设置免密
        //sshpass -p Swyt@ ssh-copy-id -p 22 -o StrictHostKeyChecking=no root@220.200.154.76
        String cmd = "sshpass -p '" + targetPassWd
                + "' ssh-copy-id -p 22 -o StrictHostKeyChecking=no " + userName +"@" + targetIp;
        //多试几次
        int retry = 0;
        while (true) {
            SshResponse response = SshUtils.runCommand(conn, cmd);
            if (response.getReturnCode() != 0) {
                LOGGER.error("setNoPassWdCopy fail targetIp={} errorMsg={} cmd={}",
                        targetIp, response.getErrOutput(), cmd);
            }else {
                break;
            }

            retry++;
            if (retry > 3){
                LOGGER.error("setNoPassWdCopy too many times targetIp={} errorMsg={} cmd={}",
                        targetIp, response.getErrOutput(), cmd);
                break;
            }
        }
    }

    /**
     *
     * @param masterUser
     * @param externalIp
     * @param passwd
     * @param targetUser
     * @param targetIp
     * @param targetPasswd
     * @throws Exception
     */
    @Override
    public void setNoPassWdCopy(String masterUser, String externalIp, String passwd, String targetUser, String targetIp, String targetPasswd) throws Exception {

        SshConnection conn  = new SshConnection(masterUser,passwd,externalIp);
        //免密
        //sshpass -p Swyt@20 ssh-copy-id -p 22 -o StrictHostKeyChecking=no root@172.16.1.159
        String cmd = "sshpass -p '" + targetPasswd
                + "' ssh-copy-id -p 22 -o StrictHostKeyChecking=no " + targetUser +"@" + targetIp;
        int retry = 0;
        while (true) {
            SshResponse response = SshUtils.runCommand(conn, cmd);
            if (response.getReturnCode() != 0) {
                LOGGER.error("setNoPassWdCopy master to worker user fail targetIp={} errorMsg={} cmd={}"
                        , targetIp, response.getErrOutput(), cmd);
            }else {
                break;
            }

            retry++;
            if (retry > 3){
                LOGGER.error("setNoPassWdCopy master to worker too many times fail targetIp={} errorMsg={} cmd={}"
                        , targetIp, response.getErrOutput(), cmd);
            }
        }
    }

    /**
     *
     * @param masterModel
     * @throws Exception
     */
    @Override
    public void tarXvfInstallPackage(VmModel masterModel) throws Exception {

        SshConnection conn  = new SshConnection(ROOT_NAME,masterModel.getPasswd(),masterModel.getExternalIp());
        //拷贝安装包
        String packageName = installPackageName;
        if (StringUtils.equalsIgnoreCase(masterModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)){
            packageName = installPackageGpuName;
        }
        String cmd = "tar -xvf /home/"+ packageName +" -C /home";
        SshResponse response = SshUtils.runCommand(conn,cmd);
        if (response.getReturnCode() != 0){
            throw new Exception("tarXvfInstallPackage fail targetIp=" + masterModel.getExternalIp() + " errorMsg=" + response.getErrOutput() + " cmd="+cmd);
        }
    }

    /**
     *
     * @param masterModel
     * @param workerModels
     * @throws Exception
     */
    @Override
    public void configInstallParamAndInstall(VmModel masterModel, List<VmModel> workerModels) throws Exception {

        SshConnection conn  = new SshConnection(ROOT_NAME,masterModel.getPasswd(),masterModel.getExternalIp());

        //修改参数，修改masterIp
        //sed -i 'N;16a\172.16.1.159' /home/CNAP-edge-x86/install/hosts-muno
        String untarName = untarPackageName;
        if (StringUtils.equalsIgnoreCase(masterModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)){
            untarName = untarPackageGpuName;
        }


        //sed -i 'N;16a 192.168.0.239' /home/CNAP-edge-x86/install/hosts-muno
        String cmdMasterIp = "sed -i 'N;16a\\" + masterModel.getInnerIp() + "' /home/"+ untarName + "/install/hosts-muno";
        SshUtils.runCommand(conn,cmdMasterIp);

        //修改参数，修改workerip
        if (!workerModels.isEmpty()) {
            for (VmModel vmModel:workerModels) {
                String cmdWorkerIp = "sed -i 'N;24a\\" + vmModel.getInnerIp() + "' /home/"+ untarName + "/install/hosts-muno";
                SshUtils.runCommand(conn,cmdWorkerIp);
            }
        }

        //修改harbor的ip
        //sed -i  '$aPORTAL_IP: 39.98.71.229' /home/CNAP-edge-x86/install/default-var.yml
        //harbor IP 为中心节点ip v1.5
        String cmdHaborIp = "sed -i '$aPORTAL_IP: " + dockerMasterUrl + "' /home/"+ untarName + "/install/default-var.yml";
        SshResponse harborIpResponse = SshUtils.runCommand(conn,cmdHaborIp);

        //GPU设置的中心节点不一样
        if (StringUtils.equalsIgnoreCase(masterModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)){

            //sed -i  '$aMECM_IP: 39.98.71.229' /home/CNAP-edge-x86-gpu/install/default-var.yml
            String gpuCmdHaborIp = "sed -i '$aMECM_IP: " + dockerMasterUrl + "' /home/"+ untarName + "/install/default-var.yml";
            SshResponse gpuHarborIpResponse = SshUtils.runCommand(conn,gpuCmdHaborIp);
        }


        int reTry = 0;
        SshResponse sshResponse = null;
        while(reTry < 2) {
            //执行安装程序
            String cmdInstall = "cd /home/" + untarName + "/install;" +
                    "ansible-playbook -i hosts-muno" +
                    " cnap_edge_muno_install.yml";
            sshResponse = SshUtils.runCommand(conn, cmdInstall);
            if (sshResponse.getReturnCode() != 0) {
                LOGGER.warn("configInstallParamAndInstall muno install failed time={} errorMsg={}",reTry,sshResponse.getStdOutput());
            }else{
                break;
            }
            reTry++;
        }
        if (sshResponse.getReturnCode() != 0){

            throw new Exception("cmdInstall fail targetIp=" + masterModel.getExternalIp() + " errormsg=" + sshResponse.getStdOutput()
                    + " ErrOutput"+ sshResponse.getErrOutput());

        }
    }

    /**
     *
     * @param masterModel
     * @return
     * @throws Exception
     */
    @Override
    public String getK8sToken(VmModel masterModel) throws Exception {

        SshConnection conn  = new SshConnection(ROOT_NAME,masterModel.getPasswd(),masterModel.getExternalIp());
        String untarName = untarPackageName;
        if (StringUtils.equalsIgnoreCase(masterModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)){
            untarName = untarPackageGpuName;
        }
        //拷贝安装包
        String cmd = "kubectl apply -f" +
                "  /home/"+ untarName + "/install/cnap-admin.yaml 1>/dev/null;echo $(kubectl get secrets $(kubectl get secret -n" +
                " kube-system|grep cnap-admin-token | awk '{print $1}') -n " +
                "kube-system -o jsonpath={'.data.token'}|base64 -d)";
        SshResponse response = SshUtils.runCommand(conn,cmd);
        if (response.getReturnCode() != 0){
            throw new Exception("tarXvfInstallPackage fail targetIp=" + masterModel.getExternalIp() + " errorMsg=" + response.getErrOutput());
        }

        return response.getStdOutput().replaceAll("\\s*", "");
    }

    /**
     *
     * @param masterModel
     * @param vmModel
     * @throws Exception
     */
    @Override
    public void delHostMumo(VmModel masterModel,VmModel vmModel) throws Exception {

        SshConnection conn  = new SshConnection(ROOT_NAME,masterModel.getPasswd(),masterModel.getExternalIp());

        String untarName = untarPackageName;
        if (StringUtils.equalsIgnoreCase(masterModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)){
            untarName = untarPackageGpuName;
        }

        //q去掉安装包配置
        String cmd = "sed -i '/" + vmModel.getInnerIp() +
                "/d' /home/"+ untarName + "/install/hosts-muno";
        LOGGER.info("delHostMumo masterIp={} tagetIp={} cmd={}",masterModel.getExternalIp(),
                        vmModel.getInnerIp(),cmd);
        SshResponse response = SshUtils.runCommand(conn,cmd);
        if (response.getReturnCode() != 0){
            LOGGER.error("delHostMumo fail cmd = " + cmd+" targetIp=" + masterModel.getExternalIp() + " errorMsg=" + response.getErrOutput());
        }
    }

    /**
     *
     * @param masterModel
     * @throws Exception
     */
    @Override
    public void installTools(VmModel masterModel) throws Exception {

        SshConnection conn  = new SshConnection(ROOT_NAME,masterModel.getPasswd(),masterModel.getExternalIp());

        String untarName = untarPackageName;
        if (StringUtils.equalsIgnoreCase(masterModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)){
            untarName = untarPackageGpuName;
        }

        //安装工具
        //cd /home/CNAP-edge-x86/;bash tools_install_offline.sh
        String cmd = "cd /home/"+ untarName + ";bash tools_install_offline.sh";
        SshResponse response = SshUtils.runCommand(conn,cmd);
        if (response.getReturnCode() != 0){
            throw new Exception("installTools offline fail targetIp=" + masterModel.getExternalIp() + " errorMsg=" + response.getErrOutput());
        }

        //gpu环境 online 安装一遍
        if (StringUtils.equalsIgnoreCase(masterModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)) {
            //online装出问题，也继续向下走
            try {
                cmd = "cd /home/" + untarName + ";bash tools_install_online.sh";
                SshResponse response02 = SshUtils.runCommand(conn, cmd);
                if (response02.getReturnCode() != 0) {
                    throw new Exception("installTools online fail targetIp=" + masterModel.getExternalIp() + " errorMsg=" + response02.getErrOutput());
                }
            }catch (Exception e){
                LOGGER.error("install tools failed offline ",e);
            }
        }
    }

    /**
     *
     * @param masterModel
     * @param vmModel
     * @throws Exception
     */
    @Override
    public void addK8sWorker(VmModel masterModel, VmModel vmModel) throws Exception {

        LOGGER.warn("begin to add k8s worker masterIp={} vmIp={} time={}",masterModel.getExternalIp(),
                vmModel.getInnerIp(),(new Date()).getTime());

        SshConnection conn  = new SshConnection(ROOT_NAME,masterModel.getPasswd(),masterModel.getExternalIp());

        //防止重试异常，删除一次host 配置
        delHostMumo(masterModel,vmModel);

        String untarName = untarPackageName;
        if (StringUtils.equalsIgnoreCase(masterModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)){
            untarName = untarPackageGpuName;
        }


        String cmdWorkerIp = "sed -i 'N;24a\\" + vmModel.getInnerIp() + "' /home/"+ untarName + "/install/hosts-muno ";
        SshResponse response = SshUtils.runCommand(conn,cmdWorkerIp);
        if (response.getReturnCode() != 0){
            LOGGER.error("addK8sWorker change hosts-muno fail cmd=" + cmdWorkerIp + " targetIp=" + vmModel.getInnerIp()
                    + " errorMsg=" + response.getErrOutput());
            throw new Exception("addK8sWorker change hosts-muno fail targetIp=" + vmModel.getInnerIp()
                    + " errorMsg=" + response.getErrOutput());
        }

        LOGGER.warn("add k8s worker finish hosts-muno config masterIp={} vmIp={} time={}",masterModel.getExternalIp(),
                vmModel.getInnerIp(),(new Date()).getTime());

        //添加配置信息
        String cmdWorkerTxt = "echo '" + vmModel.getInnerIp() + "' > /home/"+ untarName + "/install/"
        +vmModel.getInnerIp() +"workers.txt";
        SshResponse txtResponse = SshUtils.runCommand(conn,cmdWorkerTxt);
        if (txtResponse.getReturnCode() != 0){
            LOGGER.error("addK8sWorker cmdWorkerTxt fail cmd=" + cmdWorkerIp + " targetIp=" + vmModel.getInnerIp()
                    + " errorMsg=" + response.getErrOutput());
            throw new Exception("cmdWorkerTxt fail targetIp=" + vmModel.getInnerIp()
                    + " errorMsg=" + response.getErrOutput());
        }

        LOGGER.warn("add k8s worker finish worrkers.txt config masterIp={} vmIp={} time={}",masterModel.getExternalIp(),
                vmModel.getInnerIp(),(new Date()).getTime());

        String installWorker = "cd /home/"+ untarName + "/install;" +
                "ansible-playbook -i hosts-muno " +
                "cnap_edge_muno_install.yml --limit=@"  +vmModel.getInnerIp() +"workers.txt";
        //执行两次，处理有个文件锁住的问题
        SshResponse installResponse = null;
        for (int i = 0;i < 2;i++) {

            LOGGER.warn("add k8s worker install worker masterIp={} vmIp={} i={}",masterModel.getExternalIp(),
                    vmModel.getInnerIp(),i);

            installResponse = SshUtils.runCommand(conn, installWorker, true);
            if (installResponse.getReturnCode() != 0) {
                LOGGER.error("addK8sWorker install worker fail cmd={} masterIp={} targetIp={} time={} " +
                                "returnCode={} stdOutput={} errorOutput={}",
                        installWorker, masterModel.getExternalIp(), vmModel.getInnerIp(), (new Date()).getTime(),
                        installResponse.getReturnCode(), installResponse.getStdOutput(), installResponse.getStdOutput());
            }else {
                i = 3;
            }
        }

        if (installResponse.getReturnCode() != 0) {
            throw new Exception("addK8sWorker install worker fail targetIp=" + vmModel.getInnerIp()
                    + " errorMsg=" + installResponse.getErrOutput() + installResponse.getStdOutput());
        }
    }

    @Override
    public void setTencentRootSsh(VmModel masterModel) throws Exception {

        //修改root密码,腾讯云默认账户其实是ubuntu
        SshConnection conn  =
                new SshConnection("ubuntu",masterModel.getPasswd(),masterModel.getExternalIp());
        //设置root密码
        String cmd = "(echo '" + masterModel.getPasswd() + "';sleep 1;echo '"+masterModel.getPasswd()+"') | sudo passwd root;";
        SshResponse response = SshUtils.runCommand(conn,cmd);
        if (response.getReturnCode() != 0){
            throw new Exception("setTencentRootSsh changePasswd fail targetIp=" + masterModel.getExternalIp() + " errorMsg=" + response.getErrOutput());
        }

        String cmdChangeFile = "echo '" + masterModel.getPasswd() + "'| sudo -S sed -i '$a PermitRootLogin yes' /etc/ssh/sshd_config";
        SshResponse responseTelnet = SshUtils.runCommand(conn,cmdChangeFile);
        if (responseTelnet.getReturnCode() != 0){
            throw new Exception("setTencentRootSsh cmdChangeFile fail targetIp=" + masterModel.getExternalIp() + " errorMsg=" + responseTelnet.getErrOutput());
        }

        String cmdRestartFile = "echo '" + masterModel.getPasswd() + "'| sudo -S /etc/init.d/ssh restart";
        SshResponse responseRestartFile = SshUtils.runCommand(conn,cmdRestartFile);
        if  (responseRestartFile.getReturnCode() != 0){
            throw new Exception("setTencentRootSsh cmdChangeFile fail targetIp=" + masterModel.getExternalIp() + " errorMsg=" + responseTelnet.getErrOutput());
        }
    }

    @Override
    public void preInstallCmds(VmModel masterModel) throws Exception {

        SshConnection conn  = new SshConnection(ROOT_NAME,masterModel.getPasswd(),masterModel.getExternalIp());
        //拷贝安装包
        String cmd = "apt update;apt --fix-broken install -y;apt install nfs-kernel-server -y";
        //        String cmd = echo 'y'|"apt update;echo 'y'| apt --fix-broken install;echo 'y'|apt install nfs-kernel-server";
        SshResponse response = SshUtils.runCommand(conn,cmd,60);
        if (response.getReturnCode() != 0){
           LOGGER.error("preInstallCmds fail targetIp=" + masterModel.getExternalIp() + " errorMsg=" + response.getErrOutput());
        }
    }

    @Override
    public void delInnerIpNoPasswdSsh(VmModel masterModel, VmModel vmModel) {

        SshConnection conn  = new SshConnection(ROOT_NAME,masterModel.getPasswd(),masterModel.getExternalIp());
        //拷贝安装包
        String cmd = "ssh-keygen -f \"/root/.ssh/known_hosts\" -R \"" + vmModel.getInnerIp()+"\"";
        //    ssh-keygen -f "/root/.ssh/known_hosts" -R "192.168.3.27";
        SshResponse response = SshUtils.runCommand(conn,cmd,60);
        if (response.getReturnCode() != 0){
            LOGGER.error("delNoPasswdSsh fail targetIp=" + masterModel.getExternalIp() +
                    " errorMsg=" + response.getErrOutput() + " cmd="+ cmd);
        }
        LOGGER.info("delInnerIpNoPasswdSsh result masterIp={} targetIp={} code={} output={} errput={}",
                masterModel.getExternalIp(),vmModel.getInnerIp(),response.getReturnCode(),
                response.getStdOutput(),response.getErrOutput());

    }

    @Override
    public void delExternalIpNoPasswdSsh(VmModel masterModel, VmModel vmModel) {

        SshConnection conn  = new SshConnection(ROOT_NAME,masterModel.getPasswd(),masterModel.getExternalIp());
        //拷贝安装包
        String cmd = "ssh-keygen -f \"/root/.ssh/known_hosts\" -R \"" + vmModel.getExternalIp()+"\"";
        //    ssh-keygen -f "/root/.ssh/known_hosts" -R "192.168.3.27";
        SshResponse response = SshUtils.runCommand(conn,cmd,60);
        if (response.getReturnCode() != 0){
            LOGGER.error("delNoPasswdSsh fail targetIp=" + masterModel.getExternalIp() +
                    " errorMsg=" + response.getErrOutput() + " cmd="+ cmd);
        }
        LOGGER.info("delExternalIpNoPasswdSsh result masterIp={} targetIp={} code={} output={} errput={}",
                masterModel.getExternalIp(),vmModel.getExternalIp(),response.getReturnCode(),
                response.getStdOutput(),response.getErrOutput());
    }

    @Override
    public boolean checkVmNetConnectSuccess(VmModel vmModel) {
        String userName = ROOT_NAME;
        if(StringUtils.equals(vmModel.getCloudType(), CloudVendorsEnum.TencentCloud.getCloudType())){
            userName = "ubuntu";
        }

        SshConnection conn  = new SshConnection(userName,vmModel.getPasswd(),vmModel.getExternalIp());
        //拷贝安装包
        String cmd = "ls -al";
        SshResponse response = SshUtils.runCommand(conn,cmd,15);
        if (response.getReturnCode() != 0){
            LOGGER.error("checkVmNetConnectSuccess fail targetIp=" + vmModel.getExternalIp() +
                    " errorMsg=" + response.getErrOutput() + " cmd="+ cmd);
            return false;
        }
        LOGGER.info("checkVmNetConnectSuccess result masterIp={} targetIp={} code={} output={} errput={}",
                vmModel.getExternalIp(),vmModel.getInnerIp(),response.getReturnCode(),
                response.getStdOutput(),response.getErrOutput());
        return true;
    }

    @Override
    public void delNodeImage(VmModel vmModel, String imageInfo) {

        SshConnection conn  = new SshConnection(ROOT_NAME,vmModel.getPasswd(),vmModel.getExternalIp());

        String cmd = "docker rmi " + imageInfo;
        SshResponse response = SshUtils.runCommand(conn,cmd);
        if (response.getReturnCode() != 0){
            LOGGER.error("delNodeImage fail cmd={} errorMsg={}",
                    cmd,response.getErrOutput());
            return;
        }
        LOGGER.info("delNodeImage success ip={} imageInfo={}",vmModel.getExternalIp(),imageInfo);

    }

    @Override
    public String runErrorInject(VmModel vmModel, String order) {
        SshConnection conn  = new SshConnection(ROOT_NAME,vmModel.getPasswd(),vmModel.getExternalIp());

        String cmd = "cd " + errorInjectPackagePath +";" +
                "./" + order;
        SshResponse response = SshUtils.runCommand(conn,cmd);
        if (response.getReturnCode() != 0){
            LOGGER.error("stopErrorInject fail cmd={} errorMsg={}",
                    cmd,response.getErrOutput());
            return null;
        }
        LOGGER.info("runErrorInject success ip={} respones={}",vmModel.getExternalIp(),response.getStdOutput());
        String result = response.getStdOutput();
        RunInjectRetrunModel runInjectRetrunModel = JSON.parseObject(result, RunInjectRetrunModel.class);
        return runInjectRetrunModel.getResult();

    }

    @Override
    public void stopErrorInject(VmModel vmModel, String runningId) {
        SshConnection conn  = new SshConnection(ROOT_NAME,vmModel.getPasswd(),vmModel.getExternalIp());

        String cmd = "cd " + errorInjectPackagePath +";" +
                "./blade destroy " + runningId;
        SshResponse response = SshUtils.runCommand(conn,cmd);
        if (response.getReturnCode() != 0){
            LOGGER.error("stopErrorInject fail cmd={} errorMsg={}",
                    cmd,response.getErrOutput());
            return;
        }
        LOGGER.info("stopErrorInject success ip={} runningId={}",vmModel.getExternalIp(),runningId);
    }

    @Override
    public void addWorkerPrometheus(VmModel masterModel) {

        String untarName = untarPackageName;
        if (StringUtils.equalsIgnoreCase(masterModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)){
            untarName = untarPackageGpuName;
        }


        SshConnection conn  = new SshConnection(ROOT_NAME,masterModel.getPasswd(),masterModel.getExternalIp());
        //ansible-playbook -i hosts-muno cnap_edge_muno_install_add_worker_prom.yml
        String cmd = "cd /home/"+ untarName + "/install;" +
                "ansible-playbook -i hosts-muno cnap_edge_muno_install_add_worker_prom.yml";
        SshResponse response = SshUtils.runCommand(conn,cmd);
        if (response.getReturnCode() != 0){
            LOGGER.error("addWorkerPrometheus fail cmd={} errorMsg={}",
                    cmd,response.getErrOutput());
            return;
        }
        LOGGER.info("addWorkerPrometheus success resultcode={}",response.getReturnCode());

    }

    @Override
    public void copyAndInstallInjectTools(VmModel masterModel,VmModel vmModel) {

        String packageName = errorInjectPackageName;


        //说明master节点还没有把包拷贝上去
        if (masterModel == null) {
            SshConnection conn = new SshConnection(ROOT_NAME, centerPasswd, centerIp);

            //拷贝安装包
            try {
                String cmd = "sshpass -p '" + vmModel.getPasswd()
                        + "' scp " + centerInjectPackagePath + packageName + " " + ROOT_NAME + "@"
                        + vmModel.getExternalIp() + ":/home";
                SshResponse response = SshUtils.runCommand(conn, cmd);
                if (response.getReturnCode() != 0) {
                    LOGGER.error("copy master inject package fail targetIp=" + vmModel.getExternalIp() + " errorMsg=" + response.getErrOutput()
                            + " cmd=" + cmd);
                    return;
                }
            } catch (Throwable e) {
                LOGGER.error("copy master inject package error targetIp={}", vmModel.getExternalIp(), e);
                return;
            }

        }else{
            //master节点不为空，说明是要从master往worker上面拷贝包
            SshConnection conn = new SshConnection(ROOT_NAME, masterModel.getPasswd(), masterModel.getExternalIp());

            //拷贝安装包
            try {
                String cmd = "sshpass -p '" + vmModel.getPasswd()
                        + "' scp /home/"  + packageName + " " + ROOT_NAME + "@"
                        + vmModel.getExternalIp() + ":/home";
                SshResponse response = SshUtils.runCommand(conn, cmd);
                if (response.getReturnCode() != 0) {
                    LOGGER.error("copy worker inject package fail targetIp=" + vmModel.getExternalIp() + " errorMsg=" + response.getErrOutput()
                            + " cmd=" + cmd);
                    return;
                }
            } catch (Throwable e) {
                LOGGER.error("copy worker inject package error targetIp={}", vmModel.getExternalIp(), e);
                return;
            }
        }

        //到这一步vmModel上已经有包了，解压缩
        SshConnection vmConn = new SshConnection(ROOT_NAME, vmModel.getPasswd(), vmModel.getExternalIp());

        String cmd = "tar -xvf /home/"+ errorInjectPackageName +" -C /home";
        SshResponse tarResponse = SshUtils.runCommand(vmConn,cmd);
        if (tarResponse.getReturnCode() != 0){
            LOGGER.error("tar error inject package failed targetIp={} cmd={} errorMsg={}",
                    vmModel.getExternalIp(),cmd,tarResponse.getErrOutput());
            return;
        }

        //执行安装命令
        String unTarInjectPackageName = errorInjectPackagePath;
        String cmdInstall = "cd /home/" + unTarInjectPackageName +";" + errorToolsInstallCmd;
        SshResponse installSshResponse = SshUtils.runCommand(vmConn, cmdInstall);
        if (installSshResponse.getReturnCode() != 0) {
            LOGGER.warn("run install error inject tools failed vmModelIp={} errorMsg={}",
                    vmModel.getExternalIp(),installSshResponse.getStdOutput());
        }

    }
}
